<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>jdk和Hadoop搭建--普通用户下(二) | Cquang博客</title><meta name="keywords" content="jdk1.8,Hadoop 2.7,环境搭建"><meta name="author" content="Cai XianQuan"><meta name="copyright" content="Cai XianQuan"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta http-equiv="Cache-Control" content="no-transform"><meta http-equiv="Cache-Control" content="no-siteapp"><meta name="description" content="紧接上一步安装好虚拟机后，对hadoop2.7与jdk1.8的安装">
<meta property="og:type" content="article">
<meta property="og:title" content="jdk和Hadoop搭建--普通用户下(二)">
<meta property="og:url" content="https://xquan123.gitee.io/2020/10/03/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA-%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7%E4%B8%8B-%E4%BA%8C/index.html">
<meta property="og:site_name" content="Cquang博客">
<meta property="og:description" content="紧接上一步安装好虚拟机后，对hadoop2.7与jdk1.8的安装">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://xquan123.gitee.io/2020/10/03/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA-%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7%E4%B8%8B-%E4%BA%8C/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA%E9%A6%96%E9%A1%B5.jpg">
<meta property="article:published_time" content="2020-10-03T09:12:42.000Z">
<meta property="article:modified_time" content="2021-01-08T02:25:30.810Z">
<meta property="article:author" content="Cai XianQuan">
<meta property="article:tag" content="jdk1.8">
<meta property="article:tag" content="Hadoop 2.7">
<meta property="article:tag" content="环境搭建">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://xquan123.gitee.io/2020/10/03/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA-%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7%E4%B8%8B-%E4%BA%8C/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA%E9%A6%96%E9%A1%B5.jpg"><link rel="shortcut icon" href="/blog/img/favicon1.png"><link rel="canonical" href="https://xquan123.gitee.io/2020/10/03/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA-%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7%E4%B8%8B-%E4%BA%8C/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//fonts.googleapis.com" crossorigin="crossorigin"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/blog/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css"><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Titillium+Web"><script>var GLOBAL_CONFIG = { 
  root: '/blog/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":2,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"簡"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  ClickShowText: undefined,
  lightbox: 'mediumZoom',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#121212","position":"bottom-left"},
  justifiedGallery: {
    js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
    css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
  },
  isPhotoFigcaption: true,
  islazyload: false,
  isanchor: false
};

var saveToLocal = {
  set: function setWithExpiry(key, value, ttl) {
    const now = new Date()
    const expiryDay = ttl * 86400000
    const item = {
      value: value,
      expiry: now.getTime() + expiryDay,
    }
    localStorage.setItem(key, JSON.stringify(item))
  },

  get: function getWithExpiry(key) {
    const itemStr = localStorage.getItem(key)

    if (!itemStr) {
      return undefined
    }
    const item = JSON.parse(itemStr)
    const now = new Date()

    if (now.getTime() > item.expiry) {
      localStorage.removeItem(key)
      return undefined
    }
    return item.value
  }
}</script><script id="config_change">var GLOBAL_CONFIG_SITE = { 
  isPost: true,
  isHome: false,
  isHighlightShrink: true,
  isToc: true,
  postUpdate: '2021-01-08 10:25:30'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(function () {  window.activateDarkMode = function () {
    document.documentElement.setAttribute('data-theme', 'dark')
    if (document.querySelector('meta[name="theme-color"]') !== null) {
      document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
    }
  }
  window.activateLightMode = function () {
    document.documentElement.setAttribute('data-theme', 'light')
   if (document.querySelector('meta[name="theme-color"]') !== null) {
      document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
    }
  }
  const autoChangeMode = 'false'
  const t = saveToLocal.get('theme')
  if (autoChangeMode === '1') {
    const isDarkMode = window.matchMedia('(prefers-color-scheme: dark)').matches
    const isLightMode = window.matchMedia('(prefers-color-scheme: light)').matches
    const isNotSpecified = window.matchMedia('(prefers-color-scheme: no-preference)').matches
    const hasNoSupport = !isDarkMode && !isLightMode && !isNotSpecified
    if (t === undefined) {
      if (isLightMode) activateLightMode()
      else if (isDarkMode) activateDarkMode()
      else if (isNotSpecified || hasNoSupport) {
        const now = new Date()
        const hour = now.getHours()
        const isNight = hour <= 6 || hour >= 18
        isNight ? activateDarkMode() : activateLightMode()
      }
      window.matchMedia('(prefers-color-scheme: dark)').addListener(function (e) {
        if (saveToLocal.get('theme') === undefined) {
          e.matches ? activateDarkMode() : activateLightMode()
        }
      })
    } else if (t === 'light') activateLightMode()
    else activateDarkMode()
  } else if (autoChangeMode === '2') {
    const now = new Date()
    const hour = now.getHours()
    const isNight = hour <= 6 || hour >= 18
    if (t === undefined) isNight ? activateDarkMode() : activateLightMode()
    else if (t === 'light') activateLightMode()
    else activateDarkMode()
  } else {
    if (t === 'dark') activateDarkMode()
    else if (t === 'light') activateLightMode()
  }const asideStatus = saveToLocal.get('aside-status')
if (asideStatus !== undefined) {
   if (asideStatus === 'hide') {
     document.documentElement.classList.add('hide-aside')
   } else {
     document.documentElement.classList.remove('hide-aside')
   }
}})()</script><link rel="stylesheet" href="/css/VolantisTags.css"><meta name="generator" content="Hexo 5.3.0"></head><body><div id="web_bg"></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="author-avatar"><img class="avatar-img" src="/blog/img/%E5%A4%B4%E5%83%8F.gif" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/blog/archives/"><div class="headline">文章</div><div class="length-num">22</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/blog/tags/"><div class="headline">标签</div><div class="length-num">33</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/blog/categories/"><div class="headline">分类</div><div class="length-num">18</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/blog/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/blog/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/blog/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/blog/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 享受时光</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" href="/blog/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li></ul></div></div></div></div><div id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url(/blog/2020/10/03/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA-%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7%E4%B8%8B-%E4%BA%8C/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA%E9%A6%96%E9%A1%B5.jpg)"><nav id="nav"><span id="blog_name"><a id="site-name" href="/blog/">Cquang博客</a></span><span id="menus"><div id="search_button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/blog/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/blog/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/blog/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/blog/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 享受时光</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" href="/blog/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li></ul></div></div><span class="close" id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></span></span></nav><div id="post-info"><h1 class="post-title">jdk和Hadoop搭建--普通用户下(二)</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2020-10-03T09:12:42.000Z" title="发表于 2020-10-03 17:12:42">2020-10-03</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2021-01-08T02:25:30.810Z" title="更新于 2021-01-08 10:25:30">2021-01-08</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/blog/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%AD%A6%E4%B9%A0/">大数据学习</a><i class="fas fa-angle-right post-meta-separator"></i><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/blog/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%AD%A6%E4%B9%A0/%E5%A4%A7%E6%95%B0%E6%8D%AE%E9%9B%86%E7%BE%A4%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA/">大数据集群环境搭建</a></span></div><div class="meta-secondline"> <span class="post-meta-separator">|</span><span class="post-meta-wordcount"><i class="far fa-file-word fa-fw post-meta-icon"></i><span class="post-meta-label">字数总计:</span><span class="word-count">3.7k</span><span class="post-meta-separator">|</span><i class="far fa-clock fa-fw post-meta-icon"></i><span class="post-meta-label">阅读时长:</span><span>15分钟</span></span><span class="post-meta-separator">|</span><span class="post-meta-pv-cv"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="虚拟机环境准备"><a href="#虚拟机环境准备" class="headerlink" title="虚拟机环境准备"></a>虚拟机环境准备</h1><h2 id="创建虚拟机"><a href="#创建虚拟机" class="headerlink" title="创建虚拟机"></a>创建虚拟机</h2><p>参考上一节创建虚拟机：<a target="_blank" rel="noopener" href="https://www.caixianquan.tk/2020/10/02/%E5%9C%A8%E8%99%9A%E6%8B%9F%E6%9C%BA%E4%B8%AD%E5%AE%89%E8%A3%85CentOS-%E4%B8%80/">https://www.caixianquan.tk/2020/10/02/在虚拟机中安装CentOS-一/</a></p>
<h2 id="导入虚拟机"><a href="#导入虚拟机" class="headerlink" title="导入虚拟机"></a>导入虚拟机</h2><ul>
<li>从移动硬盘中导入虚拟机，这时虚拟机保存位置的文件名是不能更改的，可以直接从这台导入的虚拟机另外复制出想要的虚拟机，然后重命名，就可以将虚拟机保存的位置更改名字了。如下：<br><img src="1%E3%80%81%E5%AF%BC%E5%85%A5%E8%99%9A%E6%8B%9F%E6%9C%BA.png" alt="导入虚拟机"></li>
</ul>
<h2 id="关闭防火墙"><a href="#关闭防火墙" class="headerlink" title="关闭防火墙"></a>关闭防火墙</h2><div class="note primary flat"><p><font color=red size=3><strong>大数据工作环境一般也是关闭防火墙的，需要对外服务，再从集群与外界环境交互中增加防火墙</strong></font></p>
</div>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line">sudo service iptables stop //服务关闭</span><br><span class="line">sudo chkconfig iptables off //关闭开机自启</span><br><span class="line">service iptables status  //查看防火墙状态</span><br><span class="line">chkconfig --list iptables  //显示防火墙的运行状态信息</span><br><span class="line">等级0表示：表示关机 </span><br><span class="line">等级1表示：单用户模式 </span><br><span class="line">等级2表示：无网络连接的多用户命令行模式 </span><br><span class="line">等级3表示：有网络连接的多用户命令行模式 </span><br><span class="line">等级4表示：不可用 </span><br><span class="line">等级5表示：带图形界面的多用户模式 </span><br><span class="line">等级6表示：重新启动 </span><br><span class="line"></span><br><span class="line">CentOS7防火墙</span><br><span class="line">启动： systemctl start firewalld</span><br><span class="line">关闭： systemctl stop firewalld</span><br><span class="line">查看状态： systemctl status firewalld </span><br><span class="line">开机禁用  ： systemctl disable firewalld</span><br><span class="line">开机启用  ： systemctl enable firewalld</span><br></pre></td></tr></table></figure>

<h2 id="添加普通用户"><a href="#添加普通用户" class="headerlink" title="添加普通用户"></a>添加普通用户</h2><div class="note primary flat"><p>生产环境下需要使用普通用户来操作集群，实验环境可以使用root用户</p>
</div>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">useradd caixianquan</span><br><span class="line">passwd caixianquan</span><br></pre></td></tr></table></figure>
<p>假设虚拟机中已经存在一个普通用户，这里修改用户名及家目录</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">更改用户名：</span><br><span class="line">usermod -l 新用户名 旧用户名</span><br><span class="line">更新家目录：</span><br><span class="line">usermod -d /home/新家目录 -m 新用户名</span><br></pre></td></tr></table></figure>
<h2 id="配置这个用户为sudoers"><a href="#配置这个用户为sudoers" class="headerlink" title="配置这个用户为sudoers"></a>配置这个用户为sudoers</h2><p><font color=red size=3><strong>为用户授予能够使用root命令</strong></font></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">vim /etc/sudoers</span><br><span class="line">   在root    ALL=(ALL)       ALL</span><br><span class="line">   添加caixianquan    ALL=(ALL)       NOPASSWD:ALL</span><br><span class="line">   保存时wq!强制保存</span><br></pre></td></tr></table></figure>
<p>比如使用普通用户创建用户，则需要在前面加上sudo即可</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost opt]$ userdel -r ahhahaha</span><br><span class="line">bash: /usr/sbin/userdel: 权限不够</span><br><span class="line">[caixianquan@localhost opt]$ sudo userdel -r ahhahaha</span><br></pre></td></tr></table></figure>
<h2 id="在-opt目录下创建software和module两个文件夹，并更改所有权"><a href="#在-opt目录下创建software和module两个文件夹，并更改所有权" class="headerlink" title="在/opt目录下创建software和module两个文件夹，并更改所有权"></a>在/opt目录下创建software和module两个文件夹，并更改所有权</h2><div class="note primary flat"><p>这里主要是为后面的集群搭建设置目录以及安装位置</p>
</div>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">mkdir /opt/module /opt/software</span><br><span class="line">chown caixianquan:caixianquan /opt/module /opt/software</span><br></pre></td></tr></table></figure>
<h2 id="配置hosts文件【添加域名，后面免登录】"><a href="#配置hosts文件【添加域名，后面免登录】" class="headerlink" title="配置hosts文件【添加域名，后面免登录】"></a>配置hosts文件【添加域名，后面免登录】</h2><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">vim /etc/hosts</span><br><span class="line">=============================</span><br><span class="line">192.168.56.100   hadoop100</span><br><span class="line">192.168.56.101   hadoop101</span><br><span class="line">192.168.56.102   hadoop102</span><br><span class="line">192.168.56.103   hadoop103</span><br><span class="line">192.168.56.104   hadoop104</span><br><span class="line">192.168.56.105   hadoop105</span><br><span class="line">192.168.56.106   hadoop106</span><br><span class="line">192.168.56.107   hadoop107</span><br><span class="line">192.168.56.108   hadoop108</span><br><span class="line">192.168.56.109   hadoop109</span><br><span class="line">=============================</span><br></pre></td></tr></table></figure>
<p>小技巧：使用shell来做：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">! /bin/bash</span></span><br><span class="line">for((i = 100;i &lt; 110; i++))</span><br><span class="line">do</span><br><span class="line">        echo &quot;192.168.56.$i hadoop$i&quot; &gt;&gt; /etc/hosts</span><br><span class="line">done</span><br></pre></td></tr></table></figure>
<h2 id="改静态IP"><a href="#改静态IP" class="headerlink" title="改静态IP"></a>改静态IP</h2><p><font color=red size=3><strong>【每次克隆新虚拟机，需要从这开始修改，使用一般用户登录】</strong></font></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">DEVICE=eth0</span><br><span class="line">TYPE=Ethernet</span><br><span class="line">ONBOOT=yes</span><br><span class="line">BOOTPROTO=static</span><br><span class="line">NAME=&quot;eth0&quot;</span><br><span class="line">IPADDR=192.168.5.101</span><br><span class="line">PREFIX=24</span><br><span class="line">GATEWAY=192.168.5.2</span><br><span class="line">DNS1=192.168.5.2</span><br></pre></td></tr></table></figure>
<h2 id="修改主机名"><a href="#修改主机名" class="headerlink" title="修改主机名"></a>修改主机名</h2><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">CentOS6下：编辑vim /etc/sysconfig/network</span><br><span class="line">    改HOSTNAME=hadoop100</span><br><span class="line">CentOS下：编辑vim /etc/hostname</span><br><span class="line">        改hostname=hadoop100</span><br><span class="line">        </span><br><span class="line">或者直接使用命令：hostnamectl  set-hostname hadoop100</span><br><span class="line">【退出重新登录即可】</span><br></pre></td></tr></table></figure>
<h2 id="备份环境"><a href="#备份环境" class="headerlink" title="备份环境"></a>备份环境</h2><p>保存为快照，下次直接从这个快照克隆其他节点<br><img src="2%E3%80%81%E5%A4%87%E4%BB%BD%E7%8E%AF%E5%A2%83.png" alt="备份环境"></p>
<h1 id="SSH无密登录配置"><a href="#SSH无密登录配置" class="headerlink" title="SSH无密登录配置"></a>SSH无密登录配置</h1><h2 id="免密登录原理"><a href="#免密登录原理" class="headerlink" title="免密登录原理"></a>免密登录原理</h2><ol>
<li><p>ssh身份验证：本机A向远程主机B发送一个登录请求，B向A发送一个秘钥用来加密数据流。<br>A通过秘钥加密本机密码发送给B，然后B通过配对秘钥解密核对密码，认证成功即可登录</p>
</li>
<li><p>ssh首次连接过程中需要输入yes，这时因为远程主机在向本机发送公钥时，要确保是否是目标主机，需要人工核对。<br><img src="3%E3%80%81%E5%85%8D%E5%AF%86%E7%99%BB%E5%BD%95%E5%8E%9F%E7%90%86.png" alt="免密登录原理"></p>
</li>
</ol>
<h2 id="步骤"><a href="#步骤" class="headerlink" title="步骤"></a>步骤</h2><ol>
<li><p>生成公钥和私钥</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost ~]$ ssh-keygen -t rsa</span><br></pre></td></tr></table></figure>
<ul>
<li>然后敲（三个回车），就会生成两个文件id_rsa（私钥）、id_rsa.pub（公钥）</li>
<li>其中在~/.ssh目录下还有一个known_hosts文件，保存之前使用ssh登录过的主机ip，即首次连接时需要输入yes的过程，如果删除该文件，那么下次连接还是首次连接，得人工核对输入yes</li>
</ul>
</li>
<li><p>将公钥拷贝到要免密登录的目标机器上</p>
<div class="note primary flat"><p>注意：本身节点的authorized_keys文件也需要保存公钥</p>
</div>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost .ssh]$ ssh-copy-id hadoop103</span><br></pre></td></tr></table></figure>
<p>在目标机器上就会保存对应的公钥，文件为：authorized_keys【可以有多把公钥】<br>注意.ssh文件夹自动生成不会有权限问题</p>
</li>
<li><p>在其他hadoop103、hadoop102上</p>
<ul>
<li>也需要在各自的主机上生成对应的私钥和公钥</li>
<li>然后在hadoop103和hadoop102通过ssh-copy-id命令将公钥发送给hadoop101上，那么hadoop101上的authorized_keys文件就保存包括本身的所有公钥，直接通过xsync将authorized_keys文件分发给其他节点上即可实现所有机器免密互相登录</li>
</ul>
</li>
<li><p>.ssh文件夹下（~/.ssh）的文件功能解释<br><img src="4%E3%80%81%E6%96%87%E4%BB%B6%E8%AF%B4%E6%98%8E.png" alt="文件说明"></p>
</li>
</ol>
<h1 id="xsync集群分发脚本"><a href="#xsync集群分发脚本" class="headerlink" title="xsync集群分发脚本"></a>xsync集群分发脚本</h1><h2 id="需求"><a href="#需求" class="headerlink" title="需求"></a>需求</h2><p>循环复制文件到所有节点的相同目录下</p>
<h2 id="需求分析"><a href="#需求分析" class="headerlink" title="需求分析"></a>需求分析</h2><ol>
<li>rsync命令原始拷贝：<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">rsync  -av     /opt/module  root@hadoop103:/opt/</span><br></pre></td></tr></table></figure></li>
<li>期望脚本：<ul>
<li>xsync要同步的文件名称</li>
</ul>
</li>
<li><font color=red size=3><strong>说明：在/home/caixianquan/bin这个目录下存放的脚本，caixianquan用户可以在系统任何地方直接执行。</strong></font></li>
</ol>
<h2 id="脚本实现"><a href="#脚本实现" class="headerlink" title="脚本实现"></a>脚本实现</h2><ol>
<li><p>在/home/caixianquan目录下创建bin目录，并在bin目录下xsync创建文件，在root用户下，/root目录下创建bin目录</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop101 ~]$ mkdir bin</span><br><span class="line">[caixianquan@hadoop101 ~]$ cd bin/</span><br><span class="line">[caixianquan@hadoop101 bin]$ touch xsync</span><br><span class="line">[caixianquan@hadoop101 bin]$ vi xsync</span><br></pre></td></tr></table></figure>
<p>在该文件中编写如下代码</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">!/bin/bash</span></span><br><span class="line"><span class="meta">#</span><span class="bash">1 获取输入参数个数，如果没有参数，直接退出</span></span><br><span class="line">pcount=$#</span><br><span class="line">if ((pcount==0)); then</span><br><span class="line">echo no args;</span><br><span class="line">exit;</span><br><span class="line">fi</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash">2 获取文件名称</span></span><br><span class="line">p1=$1</span><br><span class="line">fname=`basename $p1`</span><br><span class="line">echo fname=$fname</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash">3 获取上级目录到绝对路径</span></span><br><span class="line">pdir=`cd -P $(dirname $p1); pwd`</span><br><span class="line">echo pdir=$pdir</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash">4 获取当前用户名称</span></span><br><span class="line">user=`whoami`</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash">5 循环</span></span><br><span class="line">for((host=101; host&lt;103; host++)); do</span><br><span class="line">        echo ------------------- hadoop$host --------------</span><br><span class="line">        rsync -av $pdir/$fname $user@hadoop$host:$pdir</span><br><span class="line">done</span><br></pre></td></tr></table></figure></li>
<li><p>修改脚本 xsync 具有执行权限</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop101 bin]$ chmod 777 xsync</span><br></pre></td></tr></table></figure></li>
<li><p>调用脚本形式：xsync 文件名称</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop101 bin]$ xsync /home/atguigu/bin</span><br></pre></td></tr></table></figure>
<p><font color=red size=3><strong>注意：如果将xsync放到/home/caixianquan/bin目录下仍然不能实现全局使用，可以将xsync移动到/usr/local/bin目录下【或者/bin下】。</strong></font></p>
</li>
<li><p>分发脚本到其他节点的/bin目录上</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop101 bin]$ sudo rsync -av /bin/xsync hadoop102:/bin</span><br></pre></td></tr></table></figure></li>
<li><p>将/etc/profile文件发送到其他节点上，注意要使用sudo命令【普通用户下】</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop101 opt]$ sudo xsync /etc/profile</span><br></pre></td></tr></table></figure>
<p>使用source命令执行配置文件profile</p>
</li>
</ol>
<h1 id="jpsall脚本"><a href="#jpsall脚本" class="headerlink" title="jpsall脚本"></a>jpsall脚本</h1><ol>
<li>编写jpsall脚本，实现观察多台节点的jps进程</li>
<li>在/home/caixianquan目录下创建bin目录，并在bin目录下xsync创建文件，文件内容如下：<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop101 ~]$ mkdir bin</span><br><span class="line">[caixianquan@hadoop101 ~]$ cd bin/</span><br><span class="line">[caixianquan@hadoop101 bin]$ touch jpsall</span><br><span class="line">[caixianquan@hadoop101 bin]$ vi jpsall</span><br></pre></td></tr></table></figure>
在该文件中编写如下代码<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">!/bin/bash</span></span><br><span class="line">for i in hadoop101 hadoop102 hadoop103</span><br><span class="line">do</span><br><span class="line">echo --------------------- $i -------------------------------</span><br><span class="line">ssh $i  &quot;source /etc/profile &amp;&amp; jps | grep -v Jps &quot;</span><br><span class="line">done</span><br></pre></td></tr></table></figure></li>
<li>修改脚本 jpsall具有执行权限<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop101 bin]$ chmod 777 xsync</span><br></pre></td></tr></table></figure></li>
<li>调用脚本形式：jpsall文件名称<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop101 bin]$ jpsall</span><br><span class="line">--------------------- hadoop102 -------------------------------</span><br><span class="line">5153 QuorumPeerMain</span><br><span class="line">--------------------- hadoop103 -------------------------------</span><br><span class="line">5300 QuorumPeerMain</span><br><span class="line">--------------------- hadoop104 -------------------------------</span><br><span class="line">5298 QuorumPeerMain</span><br></pre></td></tr></table></figure></li>
<li>分发脚本到其他节点的/bin目录上<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop101 ~]$ xsync bin/</span><br></pre></td></tr></table></figure>
<h1 id="安装JDK"><a href="#安装JDK" class="headerlink" title="安装JDK"></a>安装JDK</h1><h2 id="解压"><a href="#解压" class="headerlink" title="解压"></a>解压</h2></li>
<li>JDK版本：jdk-8u144-linux-x64.tar.gz</li>
<li>查询是否已安装Java【能否使用命令java】，如果有就先卸载<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost software]$ java -version</span><br><span class="line">openjdk version &quot;1.8.0_161&quot;</span><br><span class="line">OpenJDK Runtime Environment (build 1.8.0_161-b14)</span><br><span class="line">OpenJDK 64-Bit Server VM (build 25.161-b14, mixed mode)</span><br><span class="line">[caixianquan@localhost software]$ rpm -qa | grep java</span><br><span class="line">python-javapackages-3.4.1-11.el7.noarch</span><br><span class="line">java-1.8.0-openjdk-headless-1.8.0.161-2.b14.el7.x86_64</span><br><span class="line">javapackages-tools-3.4.1-11.el7.noarch</span><br><span class="line">tzdata-java-2018c-1.el7.noarch</span><br><span class="line">java-1.8.0-openjdk-1.8.0.161-2.b14.el7.x86_64</span><br><span class="line"></span><br><span class="line">【卸载java】</span><br><span class="line">[caixianquan@localhost software]$ rpm -qa | grep java | xargs sudo rpm -e --nodeps</span><br><span class="line">[caixianquan@localhost software]$ rpm -qa | grep java</span><br><span class="line">[caixianquan@localhost software]$ java</span><br><span class="line">-bash: /usr/bin/java: 没有那个文件或目录</span><br></pre></td></tr></table></figure></li>
<li>解压压缩包</li>
</ol>
<ul>
<li>将java压缩包解压到/opt/module目录下，hadoop安装包同理</li>
<li>其中参数：z、压缩编码，x、解压，v、打印信息，f、指定文件<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost software]$ tar -zxvf jdk-8u144-linux-x64.tar.gz  -C /opt/module</span><br><span class="line">[caixianquan@localhost software]$ cd /opt/module/</span><br><span class="line">[caixianquan@localhost module]$ ls</span><br><span class="line">hadoop-2.7.2  jdk1.8.0_144</span><br></pre></td></tr></table></figure>
<h2 id="配置环境变量"><a href="#配置环境变量" class="headerlink" title="配置环境变量"></a>配置环境变量</h2></li>
</ul>
<ol>
<li><p>编辑/etc/profile文件</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">如果不使用sudo命令，打开的文件是只读方式</span></span><br><span class="line">[caixianquan@localhost module]$ sudo vim /etc/profile</span><br></pre></td></tr></table></figure></li>
<li><p>往profile文件追加内容：【export为设置全局变量关键字，即当前shell和子shell都有这个文件】</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">JAVA_HOME</span></span><br><span class="line">export JAVA_HOME=/opt/module/jdk1.8.0_144</span><br><span class="line">export PATH=$PATH:$JAVA_HOME/bin</span><br></pre></td></tr></table></figure></li>
<li><p>执行环境配置文件</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost module]$ source /etc/profile</span><br></pre></td></tr></table></figure></li>
<li><p>查看配置状态</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost module]$ echo $JAVA_HOME</span><br><span class="line">/opt/module/jdk1.8.0_144</span><br><span class="line">[caixianquan@localhost module]$ java -version</span><br><span class="line">java version &quot;1.8.0_144&quot;</span><br><span class="line">Java(TM) SE Runtime Environment (build 1.8.0_144-b01)</span><br><span class="line">Java HotSpot(TM) 64-Bit Server VM (build 25.144-b01, mixed mode)</span><br></pre></td></tr></table></figure>
<h1 id="安装Hadoop"><a href="#安装Hadoop" class="headerlink" title="安装Hadoop"></a>安装Hadoop</h1><h2 id="配置环境变量-1"><a href="#配置环境变量-1" class="headerlink" title="配置环境变量"></a>配置环境变量</h2></li>
<li><p>同上述配置Java环境变量，追加内容为：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">HADOOP_HOME</span></span><br><span class="line">export HADOOP_HOME=/opt/module/hadoop-2.7.2</span><br><span class="line">export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin</span><br></pre></td></tr></table></figure></li>
<li><p>查看环境配置</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost module]$ hadoop version</span><br><span class="line">Hadoop 2.7.2</span><br><span class="line">Subversion Unknown -r Unknown</span><br><span class="line">Compiled by root on 2017-05-22T10:49Z</span><br><span class="line">Compiled with protoc 2.5.0</span><br><span class="line">From source with checksum d0fda26633fa762bff87ec759ebe689c</span><br><span class="line">This command was run using /opt/module/hadoop-2.7.2/share/hadoop/common/hadoop-common-2.7.2.jar</span><br></pre></td></tr></table></figure>
<h2 id="集群配置"><a href="#集群配置" class="headerlink" title="集群配置"></a>集群配置</h2><h3 id="集群部署规划"><a href="#集群部署规划" class="headerlink" title="集群部署规划"></a>集群部署规划</h3></li>
</ol>
<p><img src="5%E3%80%81%E9%9B%86%E7%BE%A4%E9%83%A8%E7%BD%B2%E8%A7%84%E5%88%92.png" alt="集群部署规划"></p>
<ul>
<li>DN和NM需要在同一个节点上，DN管理节点上的磁盘，NM管理节点上的CPU和内存等资源</li>
<li>一般集群的最小规模需要六台，RM、2NN、NN各自一台，DN和NM分布在三台。</li>
</ul>
<h3 id="配置集群"><a href="#配置集群" class="headerlink" title="配置集群"></a>配置集群</h3><ol>
<li>核心配置文件： 配置core-site.xml<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">在该文件中编写如下配置</span><br><span class="line">&lt;!-- 指定HDFS中NameNode的地址 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">        &lt;name&gt;fs.defaultFS&lt;/name&gt;</span><br><span class="line">      &lt;value&gt;hdfs://hadoop102:9000&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br><span class="line"> </span><br><span class="line">&lt;!-- 指定Hadoop运行时产生文件的存储目录 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">        &lt;name&gt;hadoop.tmp.dir&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;/opt/module/hadoop-2.7.2/data/tmp&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br></pre></td></tr></table></figure></li>
<li>HDFS配置文件<ul>
<li>配置hadoop-env.sh<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">export JAVA_HOME=/opt/module/jdk1.8.0_144</span><br></pre></td></tr></table></figure></li>
<li>配置hdfs-site.xml<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">在该文件中编写如下配置</span><br><span class="line">&lt;property&gt;</span><br><span class="line">        &lt;name&gt;dfs.replication&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;3&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br><span class="line"> </span><br><span class="line">&lt;!-- 指定Hadoop辅助名称节点主机配置 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">      &lt;name&gt;dfs.namenode.secondary.http-address&lt;/name&gt;</span><br><span class="line">      &lt;value&gt;hadoop104:50090&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br></pre></td></tr></table></figure></li>
</ul>
</li>
<li>YARN配置文件<ul>
<li>配置yarn-env.sh<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">export JAVA_HOME=/opt/module/jdk1.8.0_144</span><br></pre></td></tr></table></figure></li>
<li>配置yarn-site.xml<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">在该文件中增加如下配置</span><br><span class="line">&lt;!-- Reducer获取数据的方式 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">        &lt;name&gt;yarn.nodemanager.aux-services&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;mapreduce_shuffle&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br><span class="line"> </span><br><span class="line">&lt;!-- 指定YARN的ResourceManager的地址 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">        &lt;name&gt;yarn.resourcemanager.hostname&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hadoop103&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br></pre></td></tr></table></figure></li>
</ul>
</li>
<li>MapReduce配置文件<ul>
<li>配置mapred-env.sh<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">export JAVA_HOME=/opt/module/jdk1.8.0_144</span><br></pre></td></tr></table></figure></li>
<li>配置mapred-site.xml<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 hadoop]$ cp mapred-site.xml.template mapred-site.xml</span><br><span class="line">[atguigu@hadoop102 hadoop]$ vi mapred-site.xml</span><br><span class="line">在该文件中增加如下配置</span><br><span class="line">&lt;!-- 指定MR运行在Yarn上 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">        &lt;name&gt;mapreduce.framework.name&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;yarn&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br></pre></td></tr></table></figure>
<h3 id="在集群上分发配置好的Hadoop配置文件"><a href="#在集群上分发配置好的Hadoop配置文件" class="headerlink" title="在集群上分发配置好的Hadoop配置文件"></a>在集群上分发配置好的Hadoop配置文件</h3><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost hadoop-2.7.2]$ xsync etc</span><br></pre></td></tr></table></figure>
<h3 id="查看文件分发情况"><a href="#查看文件分发情况" class="headerlink" title="查看文件分发情况"></a>查看文件分发情况</h3><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop103 hadoop]$ cat /opt/module/hadoop-2.7.2/etc/hadoop/core-site.xml</span><br></pre></td></tr></table></figure>
<h3 id="配置历史服务器【先关闭集群】"><a href="#配置历史服务器【先关闭集群】" class="headerlink" title="配置历史服务器【先关闭集群】"></a>配置历史服务器【先关闭集群】</h3>为了查看程序的历史运行情况，需要配置一下历史服务器。具体配置步骤如下：</li>
</ul>
</li>
</ol>
<ul>
<li><p>配置yarn-site.xml</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">在该文件里面增加如下配置。</span><br><span class="line">&lt;!-- 历史服务器端地址 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">    &lt;name&gt;yarn.log.server.url&lt;/name&gt;</span><br><span class="line">    &lt;value&gt;http://hadoop104:19888/jobhistory/logs&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br></pre></td></tr></table></figure></li>
<li><p>配置mapred-site.xml</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">在该文件里面增加如下配置。</span><br><span class="line">&lt;!-- 历史服务器端地址 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">&lt;name&gt;mapreduce.jobhistory.address&lt;/name&gt;</span><br><span class="line">&lt;value&gt;hadoop104:10020&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br><span class="line">&lt;!-- 历史服务器web端地址 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">    &lt;name&gt;mapreduce.jobhistory.webapp.address&lt;/name&gt;</span><br><span class="line">    &lt;value&gt;hadoop104:19888&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br></pre></td></tr></table></figure></li>
<li><p>分发配置文件：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost etc]$ xsync hadoop/</span><br></pre></td></tr></table></figure></li>
<li><p>启动历史服务器，在hadoop104上启动【先重启集群后再启动历史服务器】</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop104 hadoop-2.7.2]$ sbin/mr-jobhistory-daemon.sh start historyserver</span><br></pre></td></tr></table></figure></li>
<li><p>查看历史服务器是否启动</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost hadoop-2.7.2]$ jps</span><br><span class="line">14930 NodeManager</span><br><span class="line">14710 DataNode</span><br><span class="line">15097 JobHistoryServer</span><br><span class="line">15145 Jps</span><br><span class="line">14813 SecondaryNameNode</span><br></pre></td></tr></table></figure></li>
<li><p>查看JobHistory</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">http://192.168.56.104:19888/jobhistory</span><br></pre></td></tr></table></figure>
<h3 id="配置日志的聚集【先关闭集群】"><a href="#配置日志的聚集【先关闭集群】" class="headerlink" title="配置日志的聚集【先关闭集群】"></a>配置日志的聚集【先关闭集群】</h3><p>日志聚集概念：应用运行完成以后，将程序运行日志信息上传到HDFS系统上。<br>日志聚集功能好处：可以方便的查看到程序运行详情，方便开发调试。<br>注意：开启日志聚集功能，需要重新启动NodeManager 、ResourceManager和HistoryManager。<br>开启日志聚集功能具体步骤如下：</p>
</li>
<li><p>配置yarn-site.xml</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop101 hadoop]$ vi yarn-site.xml</span><br></pre></td></tr></table></figure>
<p>在该文件里面增加如下配置。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">&lt;!-- 日志聚集功能使能 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">&lt;name&gt;yarn.log-aggregation-enable&lt;/name&gt;</span><br><span class="line">&lt;value&gt;true&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br><span class="line"></span><br><span class="line">&lt;!-- 日志保留时间设置7天 --&gt;</span><br><span class="line">&lt;property&gt;</span><br><span class="line">&lt;name&gt;yarn.log-aggregation.retain-seconds&lt;/name&gt;</span><br><span class="line">&lt;value&gt;604800&lt;/value&gt;</span><br><span class="line">&lt;/property&gt;</span><br></pre></td></tr></table></figure></li>
<li><p>分发配置文件：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@localhost etc]$ xsync hadoop/</span><br></pre></td></tr></table></figure>
</li>
<li><p>删除HDFS上已经存在的输出文件</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop101 hadoop-2.7.2]$ bin/hdfs dfs -rm -R /user/atguigu/output</span><br></pre></td></tr></table></figure></li>
<li><p>执行WordCount程序</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop101 hadoop-2.7.2]$ hadoop jar</span><br><span class="line"> share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.2.jar wordcount /user/atguigu/input /user/atguigu/output</span><br></pre></td></tr></table></figure></li>
<li><p>查看日志，如图2-37，2-38，2-39所示<br><a target="_blank" rel="noopener" href="http://192.168.56.104:19888/jobhistory">http://192.168.56.104:19888/jobhistory</a></p>
</li>
</ul>
<h3 id="集群单点启动"><a href="#集群单点启动" class="headerlink" title="集群单点启动"></a>集群单点启动</h3><p><font color=red size=3><strong><em>如果集群是第一次启动，需要格式化NameNode</em></strong></font></p>
<div class="note primary flat"><p>注意格式化之前，一定要先停止上次启动的所有namenode和datanode进程，然后再删除data和log数据，注意是所有节点的data和log数据</p>
</div>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 hadoop-2.7.2]$ hdfs namenode -format</span><br></pre></td></tr></table></figure>
<ul>
<li>启动集群<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">在hadoop102上启动NameNode</span></span><br><span class="line">[atguigu@hadoop102 hadoop-2.7.2]$ hadoop-daemon.sh start namenode</span><br><span class="line">[atguigu@hadoop102 hadoop-2.7.2]$ jps</span><br><span class="line">3461 NameNode</span><br><span class="line"><span class="meta">#</span><span class="bash">在hadoop102、hadoop103以及hadoop104上分别启动DataNode</span></span><br><span class="line">[atguigu@hadoop102 hadoop-2.7.2]$ hadoop-daemon.sh start datanode</span><br><span class="line">[atguigu@hadoop102 hadoop-2.7.2]$ jps</span><br><span class="line">3461 NameNode</span><br><span class="line">3608 Jps</span><br><span class="line">3561 DataNode</span><br><span class="line">[atguigu@hadoop103 hadoop-2.7.2]$ hadoop-daemon.sh start datanode</span><br><span class="line">[atguigu@hadoop103 hadoop-2.7.2]$ jps</span><br><span class="line">3190 DataNode</span><br><span class="line">3279 Jps</span><br><span class="line">[atguigu@hadoop104 hadoop-2.7.2]$ hadoop-daemon.sh start datanode</span><br><span class="line">[atguigu@hadoop104 hadoop-2.7.2]$ jps</span><br><span class="line">3237 Jps</span><br><span class="line">3163 DataNode</span><br></pre></td></tr></table></figure>
<h3 id="群起集群"><a href="#群起集群" class="headerlink" title="群起集群"></a>群起集群</h3></li>
</ul>
<ol>
<li><p>配置slaves，配置hadoop所有的从主机</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 hadoop-2.7.2]$ vim etc/hadoop/slaves</span><br><span class="line">删除localhost</span><br><span class="line">在该文件中增加如下内容：</span><br><span class="line">hadoop102</span><br><span class="line">hadoop103</span><br><span class="line">hadoop104</span><br></pre></td></tr></table></figure>
<div class="note warning flat"><p>注意：该文件中添加的内容结尾不允许有空格，文件中不允许有空行。</p>
</div>
</li>
<li><p>同步所有节点配置文件</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 hadoop]$ xsync slaves</span><br></pre></td></tr></table></figure></li>
<li><p>启动集群</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 hadoop-2.7.2]$ sbin/start-dfs.sh</span><br><span class="line">[caixianquan@hadoop102 hadoop-2.7.2]$ jps</span><br><span class="line">4166 NameNode</span><br><span class="line">4482 Jps</span><br><span class="line">4263 DataNode</span><br><span class="line"></span><br><span class="line">----------------------------------------------------------</span><br><span class="line">[caixianquan@hadoop103 hadoop-2.7.2]$ jps</span><br><span class="line">3218 DataNode</span><br><span class="line">3288 Jps</span><br><span class="line"></span><br><span class="line">------------------------------------------------------------</span><br><span class="line">[caixianquan@hadoop104 hadoop-2.7.2]$ jps</span><br><span class="line">3221 DataNode</span><br><span class="line">3283 SecondaryNameNode</span><br><span class="line">3364 Jps</span><br></pre></td></tr></table></figure></li>
<li><p>启动YARN</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop103 hadoop-2.7.2]$ sbin/start-yarn.sh</span><br></pre></td></tr></table></figure>
<div class="note warning flat"><p>注意：NameNode和ResourceManger如果不是同一台机器，不能在NameNode上启动 YARN，应该在ResouceManager所在的机器上启动YARN。</p>
</div>

</li>
</ol>
<h3 id="Web端查看SecondaryNameNode"><a href="#Web端查看SecondaryNameNode" class="headerlink" title="Web端查看SecondaryNameNode"></a>Web端查看SecondaryNameNode</h3><p>（a）浏览器中输入：<a target="_blank" rel="noopener" href="http://192.168.56.104:50090/status.html">http://192.168.56.104:50090/status.html</a><br>（b）查看SecondaryNameNode信息，如图所示。<br><img src="6%E3%80%81%E6%9F%A5%E7%9C%8BSecondaryNameNode%E4%BF%A1%E6%81%AF.png" alt="查看SecondaryNameNode信息"></p>
<p><img src="example.png" alt="example"><br><img src="example.png" alt="example"><br><img src="example.png" alt="example"><br><img src="example.png" alt="example"><br><img src="example.png" alt="example"></p>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined">Cai XianQuan</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://xquan123.gitee.io/2020/10/03/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA-%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7%E4%B8%8B-%E4%BA%8C/">https://xquan123.gitee.io/2020/10/03/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA-%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7%E4%B8%8B-%E4%BA%8C/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://xquan123.gitee.io" target="_blank">Cquang博客</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/blog/tags/jdk1-8/">jdk1.8</a><a class="post-meta__tags" href="/blog/tags/Hadoop-2-7/">Hadoop 2.7</a><a class="post-meta__tags" href="/blog/tags/%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA/">环境搭建</a></div><div class="post_share"><div class="social-share" data-image="/blog/2020/10/03/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA-%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7%E4%B8%8B-%E4%BA%8C/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA%E9%A6%96%E9%A1%B5.jpg" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/blog/2020/11/01/%E7%A8%8B%E5%BA%8F%E5%91%98%E5%BF%85%E7%9F%A5%E7%9A%84%E7%A1%AC%E6%A0%B8%E7%9F%A5%E8%AF%86%E5%A4%A7%E5%85%A8-%E8%AE%A1%E7%AE%97%E6%9C%BA%E6%93%8D%E4%BD%9C%E7%B3%BB%E7%BB%9F%E7%90%86%E8%AE%BA%E7%9F%A5%E8%AF%86%E6%A6%82%E5%BF%B5/"><img class="prev-cover" src="/blog/2020/11/01/%E7%A8%8B%E5%BA%8F%E5%91%98%E5%BF%85%E7%9F%A5%E7%9A%84%E7%A1%AC%E6%A0%B8%E7%9F%A5%E8%AF%86%E5%A4%A7%E5%85%A8-%E8%AE%A1%E7%AE%97%E6%9C%BA%E6%93%8D%E4%BD%9C%E7%B3%BB%E7%BB%9F%E7%90%86%E8%AE%BA%E7%9F%A5%E8%AF%86%E6%A6%82%E5%BF%B5/%E7%A8%8B%E5%BA%8F%E5%91%98%E5%BF%85%E7%9F%A5%E7%9A%84%E7%A1%AC%E6%A0%B8%E7%9F%A5%E8%AF%86%E5%A4%A7%E5%85%A8-%E8%AE%A1%E7%AE%97%E6%9C%BA%E6%93%8D%E4%BD%9C%E7%B3%BB%E7%BB%9F%E7%90%86%E8%AE%BA%E7%9F%A5%E8%AF%86%E6%A6%82%E5%BF%B5.png" onerror="onerror=null;src='/blog/img/404.jpg'"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">程序员必知的硬核知识大全--计算机操作系统理论知识概念</div></div></a></div><div class="next-post pull-right"><a href="/blog/2020/10/02/%E5%9C%A8%E8%99%9A%E6%8B%9F%E6%9C%BA%E4%B8%AD%E5%AE%89%E8%A3%85CentOS-%E4%B8%80/"><img class="next-cover" src="/blog/2020/10/02/%E5%9C%A8%E8%99%9A%E6%8B%9F%E6%9C%BA%E4%B8%AD%E5%AE%89%E8%A3%85CentOS-%E4%B8%80/%E5%9C%A8%E8%99%9A%E6%8B%9F%E6%9C%BA%E4%B8%AD%E5%AE%89%E8%A3%85CentOS7%E9%A6%96%E9%A1%B5.jpg" onerror="onerror=null;src='/blog/img/404.jpg'"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">在虚拟机中安装CentOS(一)</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span> 相关推荐</span></div><div class="relatedPosts-list"><div><a href="/blog/2020/10/02/在虚拟机中安装CentOS-一/" title="在虚拟机中安装CentOS(一)"><img class="cover" src="/blog/2020/10/02/%E5%9C%A8%E8%99%9A%E6%8B%9F%E6%9C%BA%E4%B8%AD%E5%AE%89%E8%A3%85CentOS-%E4%B8%80/%E5%9C%A8%E8%99%9A%E6%8B%9F%E6%9C%BA%E4%B8%AD%E5%AE%89%E8%A3%85CentOS7%E9%A6%96%E9%A1%B5.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-10-02</div><div class="title">在虚拟机中安装CentOS(一)</div></div></a></div></div></div></div><div class="aside_content" id="aside_content"><div class="card-widget card-info"><div class="card-content"><div class="card-info-avatar is-center"><img class="avatar-img" src="/blog/img/%E5%A4%B4%E5%83%8F.gif" onerror="this.onerror=null;this.src='/blog/img/friend_404.gif'" alt="avatar"/><div class="author-info__name">Cai XianQuan</div><div class="author-info__description"></div></div><div class="card-info-data"><div class="card-info-data-item is-center"><a href="/blog/archives/"><div class="headline">文章</div><div class="length-num">22</div></a></div><div class="card-info-data-item is-center"><a href="/blog/tags/"><div class="headline">标签</div><div class="length-num">33</div></a></div><div class="card-info-data-item is-center"><a href="/blog/categories/"><div class="headline">分类</div><div class="length-num">18</div></a></div></div><a class="button--animated" id="card-info-btn" href="javascript:;"><i class="fab fa-bookmark"></i><span>加入书签</span></a></div></div><div class="card-widget card-announcement"><div class="card-content"><div class="item-headline"><i class="fas fa-bullhorn card-announcement-animation"></i><span>公告</span></div><div class="announcement_content">一花一世界,一木一菩提╮(￣▽ ￣)╭</div></div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="card-content"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E8%99%9A%E6%8B%9F%E6%9C%BA%E7%8E%AF%E5%A2%83%E5%87%86%E5%A4%87"><span class="toc-number">1.</span> <span class="toc-text">虚拟机环境准备</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%88%9B%E5%BB%BA%E8%99%9A%E6%8B%9F%E6%9C%BA"><span class="toc-number">1.1.</span> <span class="toc-text">创建虚拟机</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%AF%BC%E5%85%A5%E8%99%9A%E6%8B%9F%E6%9C%BA"><span class="toc-number">1.2.</span> <span class="toc-text">导入虚拟机</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%85%B3%E9%97%AD%E9%98%B2%E7%81%AB%E5%A2%99"><span class="toc-number">1.3.</span> <span class="toc-text">关闭防火墙</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E6%B7%BB%E5%8A%A0%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7"><span class="toc-number">1.4.</span> <span class="toc-text">添加普通用户</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E8%BF%99%E4%B8%AA%E7%94%A8%E6%88%B7%E4%B8%BAsudoers"><span class="toc-number">1.5.</span> <span class="toc-text">配置这个用户为sudoers</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%9C%A8-opt%E7%9B%AE%E5%BD%95%E4%B8%8B%E5%88%9B%E5%BB%BAsoftware%E5%92%8Cmodule%E4%B8%A4%E4%B8%AA%E6%96%87%E4%BB%B6%E5%A4%B9%EF%BC%8C%E5%B9%B6%E6%9B%B4%E6%94%B9%E6%89%80%E6%9C%89%E6%9D%83"><span class="toc-number">1.6.</span> <span class="toc-text">在&#x2F;opt目录下创建software和module两个文件夹，并更改所有权</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%85%8D%E7%BD%AEhosts%E6%96%87%E4%BB%B6%E3%80%90%E6%B7%BB%E5%8A%A0%E5%9F%9F%E5%90%8D%EF%BC%8C%E5%90%8E%E9%9D%A2%E5%85%8D%E7%99%BB%E5%BD%95%E3%80%91"><span class="toc-number">1.7.</span> <span class="toc-text">配置hosts文件【添加域名，后面免登录】</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E6%94%B9%E9%9D%99%E6%80%81IP"><span class="toc-number">1.8.</span> <span class="toc-text">改静态IP</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BF%AE%E6%94%B9%E4%B8%BB%E6%9C%BA%E5%90%8D"><span class="toc-number">1.9.</span> <span class="toc-text">修改主机名</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%A4%87%E4%BB%BD%E7%8E%AF%E5%A2%83"><span class="toc-number">1.10.</span> <span class="toc-text">备份环境</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#SSH%E6%97%A0%E5%AF%86%E7%99%BB%E5%BD%95%E9%85%8D%E7%BD%AE"><span class="toc-number">2.</span> <span class="toc-text">SSH无密登录配置</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%85%8D%E5%AF%86%E7%99%BB%E5%BD%95%E5%8E%9F%E7%90%86"><span class="toc-number">2.1.</span> <span class="toc-text">免密登录原理</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E6%AD%A5%E9%AA%A4"><span class="toc-number">2.2.</span> <span class="toc-text">步骤</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#xsync%E9%9B%86%E7%BE%A4%E5%88%86%E5%8F%91%E8%84%9A%E6%9C%AC"><span class="toc-number">3.</span> <span class="toc-text">xsync集群分发脚本</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%9C%80%E6%B1%82"><span class="toc-number">3.1.</span> <span class="toc-text">需求</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%9C%80%E6%B1%82%E5%88%86%E6%9E%90"><span class="toc-number">3.2.</span> <span class="toc-text">需求分析</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%84%9A%E6%9C%AC%E5%AE%9E%E7%8E%B0"><span class="toc-number">3.3.</span> <span class="toc-text">脚本实现</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#jpsall%E8%84%9A%E6%9C%AC"><span class="toc-number">4.</span> <span class="toc-text">jpsall脚本</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%AE%89%E8%A3%85JDK"><span class="toc-number">5.</span> <span class="toc-text">安装JDK</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%A7%A3%E5%8E%8B"><span class="toc-number">5.1.</span> <span class="toc-text">解压</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F"><span class="toc-number">5.2.</span> <span class="toc-text">配置环境变量</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%AE%89%E8%A3%85Hadoop"><span class="toc-number">6.</span> <span class="toc-text">安装Hadoop</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F-1"><span class="toc-number">6.1.</span> <span class="toc-text">配置环境变量</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%9B%86%E7%BE%A4%E9%85%8D%E7%BD%AE"><span class="toc-number">6.2.</span> <span class="toc-text">集群配置</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E9%9B%86%E7%BE%A4%E9%83%A8%E7%BD%B2%E8%A7%84%E5%88%92"><span class="toc-number">6.2.1.</span> <span class="toc-text">集群部署规划</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E9%9B%86%E7%BE%A4"><span class="toc-number">6.2.2.</span> <span class="toc-text">配置集群</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%9C%A8%E9%9B%86%E7%BE%A4%E4%B8%8A%E5%88%86%E5%8F%91%E9%85%8D%E7%BD%AE%E5%A5%BD%E7%9A%84Hadoop%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="toc-number">6.2.3.</span> <span class="toc-text">在集群上分发配置好的Hadoop配置文件</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%9F%A5%E7%9C%8B%E6%96%87%E4%BB%B6%E5%88%86%E5%8F%91%E6%83%85%E5%86%B5"><span class="toc-number">6.2.4.</span> <span class="toc-text">查看文件分发情况</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E5%8E%86%E5%8F%B2%E6%9C%8D%E5%8A%A1%E5%99%A8%E3%80%90%E5%85%88%E5%85%B3%E9%97%AD%E9%9B%86%E7%BE%A4%E3%80%91"><span class="toc-number">6.2.5.</span> <span class="toc-text">配置历史服务器【先关闭集群】</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E6%97%A5%E5%BF%97%E7%9A%84%E8%81%9A%E9%9B%86%E3%80%90%E5%85%88%E5%85%B3%E9%97%AD%E9%9B%86%E7%BE%A4%E3%80%91"><span class="toc-number">6.2.6.</span> <span class="toc-text">配置日志的聚集【先关闭集群】</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E9%9B%86%E7%BE%A4%E5%8D%95%E7%82%B9%E5%90%AF%E5%8A%A8"><span class="toc-number">6.2.7.</span> <span class="toc-text">集群单点启动</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%BE%A4%E8%B5%B7%E9%9B%86%E7%BE%A4"><span class="toc-number">6.2.8.</span> <span class="toc-text">群起集群</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Web%E7%AB%AF%E6%9F%A5%E7%9C%8BSecondaryNameNode"><span class="toc-number">6.2.9.</span> <span class="toc-text">Web端查看SecondaryNameNode</span></a></li></ol></li></ol></li></ol></div></div></div><div class="card-widget card-recent-post"><div class="card-content"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/blog/2021/01/29/%E6%A1%86%E6%9E%B6%E5%A5%97%E8%B7%AF%E8%A7%A3%E7%AE%97%E6%B3%95/" title="框架套路解算法"><img src="/blog/2021/01/29/%E6%A1%86%E6%9E%B6%E5%A5%97%E8%B7%AF%E8%A7%A3%E7%AE%97%E6%B3%95/%E6%A1%86%E6%9E%B6%E5%A5%97%E8%B7%AF%E8%A7%A3%E7%AE%97%E6%B3%95%E9%A6%96%E9%A1%B5.jpg" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="框架套路解算法"/></a><div class="content"><a class="title" href="/blog/2021/01/29/%E6%A1%86%E6%9E%B6%E5%A5%97%E8%B7%AF%E8%A7%A3%E7%AE%97%E6%B3%95/" title="框架套路解算法">框架套路解算法</a><time datetime="2021-01-29T07:37:55.000Z" title="发表于 2021-01-29 15:37:55">2021-01-29</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2021/01/07/shell%E8%84%9A%E6%9C%AC%E5%AD%A6%E4%B9%A0/" title="shell脚本学习"><img src="/blog/2021/01/07/shell%E8%84%9A%E6%9C%AC%E5%AD%A6%E4%B9%A0/shell%E8%84%9A%E6%9C%AC%E5%AD%A6%E4%B9%A0%E9%A6%96%E9%A1%B5.jpg" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="shell脚本学习"/></a><div class="content"><a class="title" href="/blog/2021/01/07/shell%E8%84%9A%E6%9C%AC%E5%AD%A6%E4%B9%A0/" title="shell脚本学习">shell脚本学习</a><time datetime="2021-01-07T03:29:18.000Z" title="发表于 2021-01-07 11:29:18">2021-01-07</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2020/12/28/Linux%E9%97%AE%E9%A2%98%E9%9B%86%E9%94%A6/" title="Linux问题集锦"><img src="/blog/2020/12/28/Linux%E9%97%AE%E9%A2%98%E9%9B%86%E9%94%A6/Linux%E9%97%AE%E9%A2%98%E9%9B%86%E9%94%A6%E9%A6%96%E9%A1%B5.png" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="Linux问题集锦"/></a><div class="content"><a class="title" href="/blog/2020/12/28/Linux%E9%97%AE%E9%A2%98%E9%9B%86%E9%94%A6/" title="Linux问题集锦">Linux问题集锦</a><time datetime="2020-12-28T01:56:38.000Z" title="发表于 2020-12-28 09:56:38">2020-12-28</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2020/12/13/%E5%B8%B8%E7%94%A8%E7%9A%84Linux%E5%91%BD%E4%BB%A4%E9%9B%86/" title="常用的Linux命令集"><img src="/blog/2020/12/13/%E5%B8%B8%E7%94%A8%E7%9A%84Linux%E5%91%BD%E4%BB%A4%E9%9B%86/%E5%B8%B8%E7%94%A8%E7%9A%84Linux%E5%91%BD%E4%BB%A4%E9%9B%86%E9%A6%96%E9%A1%B5.jpg" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="常用的Linux命令集"/></a><div class="content"><a class="title" href="/blog/2020/12/13/%E5%B8%B8%E7%94%A8%E7%9A%84Linux%E5%91%BD%E4%BB%A4%E9%9B%86/" title="常用的Linux命令集">常用的Linux命令集</a><time datetime="2020-12-13T14:32:23.000Z" title="发表于 2020-12-13 22:32:23">2020-12-13</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2020/12/13/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0/" title="Flink入门学习"><img src="/blog/2020/12/13/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0%E9%A6%96%E9%A1%B5.png" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="Flink入门学习"/></a><div class="content"><a class="title" href="/blog/2020/12/13/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0/" title="Flink入门学习">Flink入门学习</a><time datetime="2020-12-13T10:21:57.000Z" title="发表于 2020-12-13 18:21:57">2020-12-13</time></div></div></div></div></div></div></div></main><footer id="footer" style="background-image: url(/blog/2020/10/03/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA-%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7%E4%B8%8B-%E4%BA%8C/jdk%E5%92%8CHadoop%E6%90%AD%E5%BB%BA%E9%A6%96%E9%A1%B5.jpg)"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2021 By Cai XianQuan</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text">Hi,  welcome  to  my  <a  target="_blank" rel="noopener" href="https://www.caixianquan.tk">blog</a>!</div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="translateLink" type="button" title="简繁转换">繁</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><div class="search-dialog__title" id="local-search-title">本地搜索</div><div id="local-input-panel"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div></div><hr/><div id="local-search-results"><div id="local-hits"></div><div id="local-stats"><div class="local-search-stats__hr" id="hr"><span>由</span> <a target="_blank" rel="noopener" href="https://github.com/wzpan/hexo-generator-search" style="color:#49B1F5;">hexo-generator-search</a>
 <span>提供支持</span></div></div></div><span class="search-close-button"><i class="fas fa-times"></i></span></div><div id="search-mask"></div></div><div><script src="https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js"></script><script src="/blog/js/utils.js"></script><script src="/blog/js/main.js"></script><script src="/blog/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/medium-zoom/dist/medium-zoom.min.js"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script src="/blog/js/search/local-search.js"></script><div class="js-pjax"><script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div><script src="/js/bookmark.js"></script><script defer="defer" id="fluttering_ribbon" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-fluttering-ribbon.min.js"></script><script id="click-heart" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/click-heart.min.js" async="async" mobile="true"></script></div></body></html>